# -*- coding: utf-8 -*-
import urllib2
import MySQLdb
import types
from bs4 import BeautifulSoup


def getHtmlCode(url):
    if url is None:
        return None
    response = urllib2.urlopen(url)
    if response.getcode() != 200:
        return None
    return response.read()


def getData(html_text):
    bs = BeautifulSoup(html_text, 'html.parser', from_encoding='utf-8')
    return bs


url = "http://www.freebuf.com/vuls/page/%s"
i = 1
conn = MySQLdb.connect(host='192.168.1.242', user='root',
                       passwd='cd!@#123', port=3306, db='test1', charset="utf8")
print connq
cur = conn.cursor()  # 数据库游标
# 报错:UnicodeEncodeError: 'latin-1' codec can't encode character
conn.set_character_set('utf8')
cur.execute('SET NAMES utf8;')
cur.execute('SET CHARACTER SET utf8;')
cur.execute('SET character_set_connection=utf8;')
'''while i <= 59:
    url_r = url % i
    html_1 = getHtmlCode (url_r)
    soup = BeautifulSoup(html_1,'html.parser',from_encoding='utf-8')

    node = soup.find_all('div',class_="news-info")
    for div in node:

         title = div.find('a')['title']
         sql =  "insert into t_test values(%s)"
         print title
         cur.execute(sql,[title])
         conn.commit()
    i += 1'''
url = "https://nvd.nist.gov/vuln/full-listing/2017/1"
html = getHtmlCode(url)
soup = BeautifulSoup(html, 'html.parser', from_encoding='utf-8')
node = soup.find_all('span', class_="col-md-2")
for span in node:
    id = span.text
    sql = "insert into t_id values(%s)"
    cur.execute(sql, [id])
    conn.commit()
    print id


